import scrapy
from qiubaiPro.items import QiubaiproItem

class QiubaiSpider(scrapy.Spider):
    name = 'qiubai'
    # allowed_domains = ['www.xxx.com']
    start_urls = ['https://www.qiushibaike.com/text/']

    # 1、 终端存储
    def parse(self, response):
        print(response.text)
        # 解析：作者的名称+段子内容
        div_list = response.xpath('//div[@class="col1 old-style-col1"]/div')
        all_data = [] # 存储所有解析到的数据
        for div in div_list:
            # xpath返回的是列表，但是列表元素一定是Selector类型的对象
            # extract可以将Selector对象中data参数存储的字符串提取出来
            # 1、
            # author = div.xpath('./div[1]/a[2]/h2/text()')[0].extract()
            # 2、
            author = div.xpath('./div[1]/a[2]/h2/text()').extract_first()
            # 列表调用了extract之后，则表示将列表中每一个Selector对象中data对应的字符串提取了出来
            content = div.xpath('./a/div/span//text()').extract()
            content = ''.join(content)
            # print(author, content)
            dic = {
                'author': author,
                'content': content
            }
            all_data.append(dic)

        return all_data

    # # 2、 管道存储
    # def parse(self, response):
    #     # 解析：作者的名称+段子内容
    #     div_list = response.xpath('//div[@class="col1 old-style-col1"]/div')
    #
    #     for div in div_list:
    #         # 匿名用户和普通用户名称的xpath不一样，如果有匿名用户程序就会报错
    #         author = div.xpath('./div[1]/a[2]/h2/text() | ./div[1]/span/h2/text()').extract_first()
    #         content = div.xpath('./a/div/span//text()').extract()
    #         content = ''.join(content)
    #
    #         item = QiubaiproItem()
    #         item['author'] = author
    #         item['content'] = content
    #         yield item # 将item提交给了管道